import time
import random

import requests

import lxml.etree as etree





def searchBookContent(domain):
    url = f"https://www.shicimingju.com/book/{domain}.html"

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62"
    }

    res = requests.get(url, headers=headers)

    print(res.content.decode())

    tree = etree.HTML(res.content.decode())
    # 抓取三国演绎章节
    mulu = tree.xpath("//div[@class='list']/a")

    for li in mulu:
        url = f"https://{domain}.com" + li.xpath("@href")[0]
        res = requests.get(url, headers=headers)
        tree = etree.HTML(res.content.decode())
        content = tree.xpath("//div[@class='text p_pad']/p/text()")
        print(''.join(content).strip())
        time.sleep(random.randint(1, 4))


# 抓取三国演义
searchBookContent("sanguoyanyi")